import os

# the input is a specific compressed file name, the output is approximate location of the actual data file. And followed by a forward scan 
locationLookUpDict = {}

# The input of this script is a list with trecIDs, the output is a new file the same with the input except that a new column for the ham probability has been generated based on the UWaterloo spam study.
inputDictForSpamRankFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/dictForSpamRank"
inputDictForSpamRankFileHandler = open(inputDictForSpamRankFileName,"r")

inputFileForTheActualSpamRankFileName = "/data1/team/weijiang/machine-learning-project-related/auxFiles/clueweb09spam.Fusion"
inputFileForTheActualSpamRankFilehandler = open(inputFileForTheActualSpamRankFileName,"r")

for line in inputDictForSpamRankFileHandler.readlines():
    compressedFileName = line.strip().split(" ")[0]
    beginningPosition = int(line.strip().split(" ")[1])
    if compressedFileName not in locationLookUpDict:
        locationLookUpDict[compressedFileName] = beginningPosition
    else:
        print "error"
        exit(1)

# should have 13355 tuples.
print len(locationLookUpDict)," tuples have been loaded."
print "Search Task"




# old version
# s = "clueweb09-en0000-00-00142"

inputTrainingIntermediateFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/wholeTrainingFileTemplete.txt"
inputTrainingIntermediateFileHandler = open(inputTrainingIntermediateFileName,"r")

outputTrainingIntermediateFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/wholeTrainingFileTempleteV2.txt"
outputTrainingIntermediateFileHandler = open(outputTrainingIntermediateFileName,"w")

headLine = inputTrainingIntermediateFileHandler.readline().strip()

outputTrainingIntermediateFileHandler.write(headLine + "\n")

# current version
for index,line in enumerate(inputTrainingIntermediateFileHandler.readlines()):
    print "index:",index
    # s = raw_input("Enter Valid WARC TREC ID:")
    lineElements = line.strip().split(" ")
    s = lineElements[1]
    
    key = s.split("-")[1] + "-" + s.split("-")[2]
    inputFileForTheActualSpamRankFilehandler.seek(locationLookUpDict[key],0)
    currentTrecID = ""
    while currentTrecID != s:
        elements = inputFileForTheActualSpamRankFilehandler.readline().strip().split(" ")
        #print elements
        currentTrecID = elements[1]
        currentTrecIDProbability = elements[0]
    
    # currentTrecID,currentTrecIDProbability
    outputTrainingIntermediateFileHandler.write(lineElements[0] + " " + lineElements[1] + " " + lineElements[2] + " " + currentTrecIDProbability + " " + lineElements[3] + "\n")
#print key,locationLookUpDict[key]


inputDictForSpamRankFileHandler.close()
inputFileForTheActualSpamRankFilehandler.close()
inputTrainingIntermediateFileHandler.close()
outputTrainingIntermediateFileHandler.close()


'''
inputFileName = "/data1/team/weijiang/machine-learning-project-related/trec-related/trecCombine09_11/priorityWarcDocumentsIDs_all_sorted.txt"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data1/team/weijiang/machine-learning-project-related/trec-related/trecCombine09_11/priorityWarcDocumentsIDs_all_sorted_with_index_number.txt"
outputFileHandler = open(outputFileName,"w")

for index,line in enumerate(inputFileHandler.readlines()):
    outputFileHandler.write(str(index) + " " + line.strip() + "\n")

inputFileHandler.close()
outputFileHandler.close()
print "Done."
'''

'''
new_directory_base = "/data5/team/weijiang/human_judge_web_pages/"

for i in range(0,134):
    dirName = "en%04d" % i
    new_directory = new_directory_base + dirName
    print new_directory
    os.mkdir(new_directory)


for i in range(0,4):
    dirName = "enwp%02d" % i
    new_directory = new_directory_base + dirName
    print new_directory
    os.mkdir(new_directory)
'''